# -*- coding: utf-8 -*-
"""

TradeZone Client

Based on Example included in TradeZone Data Center (TZDC) docs.

Important decisions about data:
  - ParseTimestamp1() replaces time with 23:59 when the period is not Min1
  - ParseTimestamp1() is triming seconds to zero.
  - ParseHistory() skips AfterMarket and Auction records that are NOT FROM TODAY.

To defeat urllib2's intrinsic buffering, you could do:
    import socket
    socket._fileobject.default_bufsize = 0
because it's actualy socket._fileobject that buffers underneath. No data will be
lost anyway, but with the default buffering (8192 bytes at a time) data may end up 
overly chunked for real-time streaming purposes (completely removing the buffering
might hurt performance, but you could try smaller chunks). [2]


References:
  [1] C# Example included in TradeZone Data Center (TZDC) installation.
  [2] http://stackoverflow.com/questions/1598331/how-to-read-continous-http-streaming-data-in-python

"""

import datetime
import time
#import dateutil.parser
import urllib2 as ul
import Pyro4
import string
import logging
import sys
import numpy as np
from threading import RLock
import threading

from misc.misc import enum
from misc import timedata
from misc import misc
import misc.quoteperiod as qp
from system import infogetter
from system import threads
from system import rtclasses

# Base socket and HTTPClient.__Get() timeout
# TODO: synchonize timeout with database server because it is specified there the Pyro commit timeout
TIMEOUT = 60
# Maximum number of tries for some operations that insist with TZDC
MAXTRIES = 7
import socket
socket.setdefaulttimeout(TIMEOUT) # Hopefully this will reflect in read() below
socket._fileobject.default_bufsize = 64 
# This option is useful for debugging. If the flag is set, the interface methods
# Will propagate exceptions
FLAG_RAISE_EXCEPTIONS = False

#############################################################################################################
## Realtime stream reader/poster

class Streamer(threads.LoopThread):
    """Reads stream from the TradeZone server and posts on the Pyro Event Server"""
    ## Setup
    # HTTPClient. MUST be set
    tz = None
    # Reference time (in seconds) between posts to the Event Server
    cycleTime = 5
    # Number of characters to read from stream each time reading is attempted
    biteSize = 512
    
    ############################################################################
    ## Interface routines
    
    @misc.CatchException
    def Pause(self, flag):
        """Use this to start/stop streaming."""
        self.logger.info("Pause(): flag=%s" % flag)
        if flag == self._flagRunning:
        # i.e., I can't stop if already stopped nor start if already started
            self._flagToPause = flag
            self._flagToResume = not flag
            self.logger.info("_flagToPause=%s; _flagToResume=%s" % (self._flagToPause, self._flagToResume))
        else:
            self.logger.info("Pause() won't do anything because _flagRunning=%s" % self._flagRunning)

    @misc.CatchException
    def SetNames(self, names):
        """Sets stock names (TZDC limits to 20). MAY RAISE EXCEPTION."""
        self._SetNames(names)
        
    def GetNames(self):
        return self._GetNames()

    @misc.CatchException
    def GetQuotess(self):
        """Will return self._quotess, whichever it has, and reset it."""
        with self.L_quotess:
            self._TranslateToNames()
            r = self._quotess
            self._quotess = {}
            return r

    def Ping2(self):
        """Returns "pong". LoopThread already has a Ping(), hence Ping2()."""
        return "pong"


    ############################################################################
    ############################################################################
    
    ## Overwritten routines

    def __init__(self):
        threads.LoopThread.__init__(self, name="TZ Streamer")
        self._tic = -1 # Last time that quotes were published
        self._urlFile = None
        # Note that it is created in a state in which it will
        # wait for a Pause(False) call
        self._flagToPause = False
        self._flagToResume = False
        self._flagRunning = False
        self._loner = "" # Unfinished line read from stream
        self._quotess = {} # Where read quotes are accumulated until read
        self._names = [] # Stock names
        self._symbols = [] # Stock symbols (synchnonized with _names)
        
        self.L_rt = threading.Lock() # Protects _symbols and _names
        self.L_quotess = threading.Lock() # Protects _quotess

    def _BeforeExit(self):
        self._CloseIfOpen()

    # *-*-*-*-*-*-*-*
    def _Cycle(self):
        try:
            # "Unusual" actions
            if self._flagToResume:
                self._AddAll()
                self._urlFile = ul.urlopen(self._GetUrl(), timeout=10)
                self._flagRunning = True
                self._flagToResume = False
            if self._flagToPause:
                self._RemoveAll()
                self._CloseIfOpen()
                self._flagRunning = False
                self._flagToPause = False
        except:
            # Here, will log exception, wait for a bit, then retry
            self.logger.exception("Failed to pause/resume")
            self.Sleep(2)
            return

        # "Normal" operation
        if self._flagRunning:
            self._ReadStream() # This does not raise exception
            self.Sleep(0.1) # Calm down


    ###########################################################################
    ## Symbols management routines

    def _GetNames(self):
        with self.L_rt:
            return self._names

    def _AddAll(self):
        """Adds all _symbols. MAY RAISE EXCEPTION."""
        with self.L_rt:
            for symbol in self._symbols:
                self._tz_AddSymbol(symbol)
            
    def _RemoveAll(self):
        """Removes all _symbols. MAY RAISE EXCEPTION."""
        with self.L_rt:
            for symbol in self._symbols:
                self._tz_RemoveSymbol(symbol)
    
    def _SetNames(self, names):
        """Action depends on _flagRunning. MAY RAISE EXCEPTION.
        
        If _flagRunning is true, adds/Removes symbols from TZDC straight away,
        plus updates _symbols and _names. Otherwise, does the latter only."""
        self.logger.debug("_SetNames(): got these names: %s" % names)
        with self.L_rt:
            symbols = [misc.Name2Symbol(x) for x in names]
            if self._flagRunning:
                self.logger.debug("_SetNames(): Gonna register names symbols WHILE RUNNING")
                # Registers only if running, otherwise will keep for later
                toAdd = set(symbols)-set(self._symbols)
                toRemove = set(self._symbols)-set(symbols)
                self.logger.debug("_SetNames(): toAdd=%s; toRemove=%s" % (toAdd, toRemove))
                for s in toRemove:
                    self._tz_RemoveSymbol(s)
                for s in toAdd:
                    self._tz_AddSymbol(s)
            self._names = names
            self._symbols = symbols
            self.logger.debug("_SetNames(): went OK: %s" % self._names)


    def _TranslateToNames(self):
        """Replace dictionary keys (which are symbols) with stock names.
        
        _quotess becomes unusable internally after this."""
        
        def _Symbol2Name(symbol):
            """Converts symbol to name by searching symbol within _symbols."""
            if symbol not in self._symbols:
                return None
            return self._names[self._symbols.index(symbol)]

        with self.L_rt:
            kk = self._quotess.keys()
            for symbol in kk:
                name = _Symbol2Name(symbol)
                if name is None:
                    self.logger.critical('Could not resolve name for symbol "%s", ignoring...' % symbol)
                    continue
                #self.logger.debug('%s ====> %s' % (symbol, name))
                self._quotess[name] = self._quotess.pop(symbol) # Replaces key

    def _tz_AddSymbol(self, s):
        self.logger.info("Gonna ADD symbol ``%s``" % s)
        self.tz._rt_AddSymbol(s)

    def _tz_RemoveSymbol(self, s):
        self.logger.info("Gonna REMOVE symbol ``%s``" % s)
        self.tz._rt_RemoveSymbol(s)
        
    ###########################################################################
    ## URL I/O; _quotess routines

    def _CloseIfOpen(self):
        """Closes _urlFile if it is open."""
        if not misc.isempty(self._urlFile):
            self._urlFile.close()
    
    def _GetUrl(self):
        """Returns stream URL."""
        return URLTemplates.GetRealtime.format(self.tz.loginId)

    def _ReadStream(self):
        """Attemts to read biteSize bites. Treats its own exceptions."""
        try:
            self.logger.debug("Going for reading %d bytes..." % self.biteSize)
            s = self._urlFile.read(self.biteSize)
            self.logger.debug("What I read (%d bytes) was: %s" % (len(s), s.__str__()))
            s = self._loner+s # Joins with previous partial line
            self._loner = "" # Resets previous partial line
            bHas = True
            if s[-2:] != "\r\n":
                if not "\r\n" in s:
                    self._loner = s
                    bHas = False
                else:
                    splitPoint = s.rfind("\r\n")
                    self._loner = s[splitPoint+2:]
                    s = s[0:splitPoint]
            if bHas:
                with self.L_quotess:
                    self._quotess = ParseTicks(s.split("\r\n"), self._quotess)
        except socket.timeout:
            self.logger.debug("Timeout, nevermind...")
        except Exception as E:
            self.logger.exception("Error reading stream")


#############################################################################################################
## HTTP Client

def GetHTTPClientProxy():
    o = Pyro4.Proxy("PYRONAME:tradezone.HTTPClient")
    try:
        o.loginId
    except:
        raise ProxyError("Could not get a valid proxy to the HTTPClient object")
    return o

def GetStreamerProxy():
    o = Pyro4.Proxy("PYRONAME:tradezone.Streamer")
    try:
        o.loginId
    except:
        raise ProxyError("Could not get a valid proxy to the Streamer object")
    return o

class HTTPClient(infogetter.InfoGetter):
    """HTTP client for the TradeZone Data Center Server.
    
    This module is NOT thread-safe but is probably being accessed by at least
    two threads. Since it does not write to the database, it became minor
    concern. Besides, the communication with the TZ Data Center is already
    very unstable."""
    
    ## Setup variables
    # Number of tries of __Get() method.
    numTries = 7

    ## State variables
    loginId = ""
    isLogged = False
    # Whether realtime streaming is running 
    flagRealtime = False
    # Realtime symbols, maintained by rt_SetSymbols()
    symbols = []
    # Streamer object
    streamer = None
    
    ## Internal
    # symbols actually added, maintained by _rt_SetSymbols()
    _symbols = []

    
    def __init__(self):
        self.logger = logging.Logger(name="TZHTTPClient", level=logging.DEBUG)
        fmtr = logging.Formatter(fmt='%(levelname)s:%(message)s')
        ch = logging.FileHandler(filename="tradezone.log")
        ch.setFormatter(fmtr)
        self.logger.addHandler(ch)
        ch = logging.StreamHandler(sys.stdout)
        ch.setFormatter(fmtr)
        self.logger.addHandler(ch) # Logs both to file and console
        
        self.Ev_login = threading.Event() # This Event is managed by Login()
        # Events are initially set so that wait() calls don't block (initially)
        self.Ev_login.set() 
        # This Event is managed by _IncCounter()() and _DecCounter()
        self.Ev_counter = threading.Event() 
        self.Ev_counter.set()
        # This lock is used to make incrementing and decrementing the counter
        # thread-safe
        self.L_counter = threading.Lock()
        # Counts how many "other" methods are currently entered
        self._counter = 0


    def _IncCounter(self):
        with self.L_counter:
            self._counter += 1
            # This will keep Login() from entering
            self.Ev_counter.clear()
        
    def _DecCounter(self):
        with self.L_counter:
            self._counter -= 1
            if self._counter <= 0:
                # This will allow Login() from entering again
                self.Ev_counter.set()



                
    ###########################################################################
    ## These routines use __Get(), so they are controlled as the others below
        
    def _rt_AddSymbol(self, symbol):
        self.Ev_login.wait()
        self._IncCounter()
        try:
            url = URLTemplates.AddRealtime.format(self.loginId, symbol)
            i = 0
            flagOk = False
            while i < MAXTRIES: # Loop to insist on Realtime, not Delay
                # Example of reply: "PETR4=DL OK" <-- delay, not wanted, BTW
                a = self.__Get(url)
                if len(a) > 0:
                    s = a[0]
                    if not ("=RT" in s): # or "=DL" in s): #TODO temporary accepting delay
                        self.logger.warning("(try %d) rt_AddSymbol() got \"%s\"" % 
                            (i, s)) # same message, currently
                    else:
                        flagOk = True
                        break
                i += 1
            if not flagOk:
                raise RuntimeError("Couldn't add symbol %s, gave up after %d tries"%
                    (symbol, i))
            self.logger.debug("rt_AddSymbol(): success: %s" % symbol)
        finally:
            self._DecCounter()
        
    def _rt_RemoveSymbol(self, symbol):
        self.Ev_login.wait()
        self._IncCounter()
        try:
            url = URLTemplates.RemoveRealtime.format(self.loginId, symbol)
            s = self.__Get(url)
            self.logger.debug("rt_RemoveSymbol(): success: %s" % symbol)
        finally:
            self._DecCounter()

    ###########################################################################

    def AssertLogged(self):
        if self.isLogged is False:
            raise HTTPClientError("Not logged, needs to log in first")

    def GetLoginURL(self):
        return URLTemplates.Login.format(
            "A762C7B7-AB84-485B-8D09-4EB948572348", "2.0.0.0", "Navala")

    def __Get(self, url, timeout=TIMEOUT):
        """
        General HTTP GET.
        
        Returns a list containing the lines of what was returned by the HTTP
        server. Tries a number of times
        
        TODO: URL can be a lambda function so that error recovery could attempt a new read with a new loginId (this would be cool).
        
        """
        while True:
            try:
                url_file = ul.urlopen(url, timeout=timeout)
                break
            except ul.HTTPError as E:
                if E.code == 403 and not "login" in url:
                    self.logger.warning("Got error 403, will reattempt login")
                    self.__Login()
                    raise # Still has to raise because loginID has changed
                else:
                    self.logger.critical("URL: "+url)
                    raise
            except:
                self.logger.critical("URL: "+url)
                raise
        # print(url_file.geturl())
        # print(url_file.info())
        i = 1
        flagOk = False
        while i <= self.numTries:
            try:
                a = url_file.readlines()
#                for aa in a:
#                    self.logger.debug(aa
                flagOk = True
            except Exception as E:
                i += 1
                if i > self.numTries:
                    self.logger.debug("Gave up after %d tr%s" % (i-1, "y" if i-1 == 1 else "ies"))
                    raise E
                self.logger.exception(E)
                self.logger.debug("Going for try %d/%d for url %s" % (i, self.numTries, url))
            if flagOk:
                break
        return a

    @misc.CatchException
    def Login(self):
        self.Ev_counter.wait() # Waits until al other methods have exited
        self.Ev_login.clear() # Blocks all other methods from entering again
        # Increases this counter even here because
        # I don't want Login() to be entered twice
        self._IncCounter()
        try:
            self.__Login()
        finally:
            self.Ev_login.set()
            self._DecCounter()

    def __Login(self):
        self.logger.debug("Logging...")
        url = self.GetLoginURL()
        a = self.__Get(url, timeout=15)
        if "LOCAL" in a[0]:
            raise RuntimeError("I don't want to log in locally")
        self.loginId = a[0]
        self.logger.info("LoginID=%s" % self.loginId)
        self.isLogged = True
        self.logger.info("Login(): success")


    def GetLoginId(self):
        self.Ev_login.wait()
        self._IncCounter()
        try:
            return self.loginId
        finally:
            self._DecCounter()
        
    @misc.CatchException
    def Logout(self):
        self.Ev_login.wait()
        self._IncCounter()
        try:
            self.logger.debug("Logging out...")
            self.AssertLogged()
            url = URLTemplates.Logout.format(self.loginId)
            self.__Get(url)
            self.loginId = ""
            self.isLogged = False
            self.logger.debug("Logout(): success")
        finally:
            self._DecCounter()
    
    @misc.CatchException
    def GetSymbols(self, pattern=""):
        """Gets list of symbols.
        
        pattern="" Parameter
        """
        self.Ev_login.wait()
        self._IncCounter()
        try:
            self.logger.debug("GetSymbols() start")
            url = URLTemplates.GetSymbols.format(self.loginId, pattern)
            a = self.__Get(url)
            result = ParseSymbols(a)
            self.logger.debug("GetSymbols() finish (%d symbols)" % len(result))
            return result
        finally:
            self._DecCounter()
                
    @misc.CatchException
    def GetHistory(self, symbol, from_=datetime.date(2013, 1, 16),
                   to=datetime.date(2013, 1, 16),
                   period=qp.min1):
        """
        Returns historical data as a timedata.Quotes object
        
        Arguments:
            symbol -- e.g. "PETR4"
            from_, to -- datetime.datetime, or string "YYYY-MM-DD HH:MM"
            period=quoteperiod.min1 -- values from quoteperiod module
        """
        self.Ev_login.wait()
        self._IncCounter()
        try:
            # Converts "from" to string, if necessary
            from_ = misc.dt2str(from_, False)
            to = misc.dt2str(misc.AddTimeIfZero(to), False)
            sPeriod = "1" if period == qp.min1 \
                      else qp.data[period]["english"]
            self.logger.debug("GetHistory() start: %s - %s to %s" % (symbol, from_, to))
            url = URLTemplates.GetHistory.format(self.loginId, symbol, from_,
                  to, sPeriod, "0") # "maxbars" is always zero
            a = self.__Get(url)
            # Result is reversed because TradeZone returns in decreasing time order
            #self.logger.debug("A HAS %d lines" % len(a))
            result = ParseHistory(a[-1::-1], period)
            s1 = misc.dt2str(misc.ts2dt(result.timestamp[0]), False) if len(result) > 0 else "?"
            s2 = misc.dt2str(misc.ts2dt(result.timestamp[-1]), False) if len(result) > 0 else "?"
            self.logger.debug("GetHistory() finish (%d quotes): %s to %s" % (len(result), s1, s2))
            return result
        finally:
            self._DecCounter()

    @misc.CatchException
    def AddMini(self, symbol):
        # TODO: parse
        self.Ev_login.wait()
        self._IncCounter()
        try:
            url = URLTemplates.AddMini.format(self.loginId, symbol)
            s = self.__Get(url)
        finally:
            self._DecCounter()
        
    @misc.CatchException
    def RemoveMini(self, symbol):
        # TODO: parse
        self.Ev_login.wait()
        self._IncCounter()
        try:
            url = URLTemplates.RemoveMini.format(self.loginId, symbol)
            s = self.__Get(url)
        finally:
            self._DecCounter()


#############################################################################################################
# Types

# Tick type
TickType = enum('Full', 'History', 'TimeStamp')
# Minibook type
MinibookType = enum('Full', 'History', 'Compact')

class Tick(object):
    """Tick class
    """
    # string; Symbol name
    Stock = None
    # timestamp
    Timestamp = None
    # unsigned int
    PreviousOrderNumber = None
    # unsigned int    
    OrderNumber = None
    # unsigned int                        
    TradeVolume = None
    # unsigned int                        
    TotalVolume = None
    # float
    TradeFinancialVolume = None
    # float
    TotalFinancialVolume = None
    # float
    Open = None
    # float
    High = None
    # float
    Low = None
    # float
    PreviousClose = None
    # float
    Last = None
    # String
    Status = " ";
    # Possible values are within TickType
    Type = TickType.TimeStamp

class Minibook(object):
    """Minibook class
    """
    # string
    Stock = None
    # List of MinibookItem objects
    Items = []
    # string
    Status = None
    # Possible values within MinibookType
    Type = MinibookType.Full
    
class MinibookItem(object):
    """Minibook item: elements of Minibook.Items
    """
    # byte
    Order = None
    # ulong 
    TotalQuantityBids = None
    # uint
    NumberOfBids = None
    # float
    BestBidPrice = None
    # float
    BestAskPrice = None
    # uint
    NumberOfAsks = None
    # ulong
    TotalQuantityAsks = None

#############################################################################################################
# Parsers

def MakeFloat(s):
    return float(string.replace(s.strip(), ",", "."))

def MakeInt(s):
    return int(float(string.replace(s.strip(), ",", ".")))

def ParseTimestamp1(s, period):
    """Parses date-time string from "gethistory" into a datetime.datetime object."""
    dt = datetime.datetime.strptime(s.strip(), 
         "%d/%m/%Y %H:%M:%S") #.replace(second=0)
    if period >= qp.daily:
        dt = dt.replace(hour=23, minute=59, second=0)
    return (dt, misc.dt2ts(dt))

def ParseTimestamp2(s):
    """Parses date-time string from "getrealtime" into a datetime.datetime object."""
    dt = datetime.datetime.strptime(s.strip(), "%Y-%m-%d %H:%M:%S") #.replace(second=0) 
    return misc.dt2ts(dt)

def ParseSymbols(rows):
    """
    Parses a string returned by the TradeZone server following the "getsymbols"
    command.
    
    """

    a = []
    for row in rows:
        if len(row) == 0:
            continue
        ff = row.split(";")
        oi = infogetter.SymbolsItem()
        oi.symbol = ff[0].strip()
        oi.description = ff[1].strip()
        oi.market = ff[2].strip()
        oi.flagDivide1000 = ff[3] == "True"
        oi.group = ff[4].strip()
        oi.supermarket = ff[5].strip()
        
        a.append(oi)

    return a

def ParseHistory(rows, period):
    """
    Parses rows returned by the TradeZone server following the "gethistory"
    command.
    
    period influences time rounding: non-intraday ticks should be given
    always with the same time (See ParseTimestamp1())
    """
    # Símbolo ; Nome ; Mercado ; Divide por mil ; Grupo ; Bolsa CR/LF

    dToday = datetime.datetime.today().date()
    numRows = len(rows)
    a = timedata.Quotes()
    a.SetSize(numRows) # Hope that setting size, then eventually shrinking is faster
                       # than appending.
    cnt = 0 # Number of rows actually added
    for row in rows:
        if len(row) == 0:
            continue
        ff = row.split(";")
        # Parses date+time and status first to decide whether will skip or not
        (dt, ts) = ParseTimestamp1(ff[0], period)
        if cnt > 0 and ts == a.timestamp[cnt-1]:
            # Tradezone is sending me some repeated quotes for some reason
            continue
        status = timedata.QuoteStatus.Regular
        temp = ff[8].strip()
        if temp == "A":
            status = timedata.QuoteStatus.AfterMarket
        elif temp == "L":
            status = timedata.QuoteStatus.Auction
        if status != timedata.QuoteStatus.Regular and dt.date() != dToday:
            continue

        a.timestamp[cnt] = ts
        a.open[cnt] = MakeFloat(ff[1])
        a.high[cnt] = MakeFloat(ff[2])
        a.low[cnt] = MakeFloat(ff[3])
        a.close[cnt] = MakeFloat(ff[4])
        #a.Qtt = MakeInt(ff[5])
        a.volume[cnt] = MakeInt(ff[6])
#####        print("MERDA --> %s; %.2g; %.2g; %.2g; %.2g; %d" % (misc.ts2dt(ts), a.open[cnt], 
#####            a.high[cnt], a.low[cnt], a.close[cnt], a.volume[cnt]))
        a.numTrades[cnt] = MakeInt(ff[7])
        a.status[cnt] = status
        a.flagRealtime[cnt] = False
        cnt += 1
    if cnt < numRows:
        a.SetSize(cnt)
    return a

def ParseMinibook(s):
    """Returns a Minibook object
    """
#    CultureInfo ci = new CultureInfo("pt-BR", false);
    mt = MinibookType.History
    if s.startswith("HM"):
        mt = MinibookType.History
    elif s.startswith("FM"):
        mt = MinibookType.Full
    elif s.startswith("CM"):
        mt = MinibookType.Compact
    else:
        return None

    s = s[3:]
    cols = s.Split(';')

    m = Minibook()
#    m.Items = new List<MinibookItem>()
    m.Type = mt
    m.Stock = cols[0].strip()
    m.Status = cols[1].strip()

    for i in range(2, len(cols)):
        s1 = cols[i].Split('=');
        if (len(s1) != 2):
            continue
        order = s1[0]
        fields = s1[1][1:-3].split(':')
        mi = MinibookItem()
        mi.order = MakeInt(order)

        mi.BestAskPrice = MakeFloat(fields[0])
        mi.NumberOfAsks = MakeInt(fields[1])
        mi.TotalQuantityAsks = MakeInt(fields[2])
        mi.BestBidPrice = MakeFloat(fields[3])
        mi.NumberOfBids = MakeInt(fields[4])
        mi.TotalQuantityBids = MakeInt(fields[5])
        
        m.Items.append(mi)

#    m.Items.Sort(delegate(MinibookItem m1, MinibookItem m2) { return m1.order.CompareTo(m2.order); });
    return m




def ParseTicks(rows, a):
    """Returns a dictionary containing symbols as keys and timedata.Quotes as values
    
    Arguments:
      rows -- result of a "getrealtime" command
      a -- already existing dictionary to append information
        
    Returns:
      {"symbol": timedate.Quotes, ...}
      Note that the symbol is just "PETR4" or "VALE5", and not "BOVESPA.VISTA.PETR4"
            
    Full tick example:
      0 FT=PETR4;      Full Tick = Symbol
      1 2013-03-06;    date
      2 17:53:07;      time
      3 16,56;         yesterday's close
      4 17,6;          today's open
      5 17,69;         today's maximum
      6 17,22;         today's minimum
      7 17,66;         today's last value
      8 100;           volume (number of papers) (probably of last trade)
      9 46292400;      total volume 
      10 1766;         volume in money (probably of last trade)
      11 8,104275E+08; total volume in money (I suppose)
      12 48664;        previous record id
      13 48665;           current record id
    """
    for s in rows:
        if len(s) <= 2:
            continue # Empty line
        type_ = s[0:2]
        if type_ in ("TS", "HT"):
            # timestamp and history ticks. Valid type_ is "FT" (Full Tick)
            continue

        s = s[3:]
        cols = s.split(';')

        stock = cols[0].strip()
        if not a.has_key(stock):
            q = timedata.Quotes()
            a[stock] = q
        else:
            q = a[stock]

        ts = ParseTimestamp2(cols[1].strip()+" "+cols[2].strip())
        last, numTrades = map(lambda x: MakeFloat(cols[x]), [7, 8])
        volume = MakeInt(cols[10])
        i = misc.BSearch(q.timestamp, ts)
        if i == -1:
            # New quote
            i = len(q)
            q.SetSize(i+1)
            q.timestamp[i] = ts
            q.open[i] = last
            q.close[i] = last
            q.high[i] = last
            q.low[i] = last
            q.numTrades[i] = numTrades
            q.volume[i] = volume
            q.flagRealtime[i] = True # TODO: temporary
            status = timedata.QuoteStatus.Regular        
            if len(cols) >= 15:
                temp = cols[14].strip()
                if temp == "A":
                    status = timedata.QuoteStatus.AfterMarket
                elif temp == "L":
                    status = timedata.QuoteStatus.Auction
            q.status[i] = status
        else:
            # Quote update
            q.close[i] = last
            q.high[i] = max(q.high[i], last)
            q.low[i] = min(q.low[i], last)
            q.numTrades[i] += numTrades
            q.volume[i] += volume
    return a



#############################################################################################################
# URL Templates

class URLTemplates(object):
    """Templates for the HTTP GET strings to communicate with the TZDataCenter
    """
    Server = "http://localhost:31008"

    Login = Server + "/login?product={0}&version={1}&description={2}";
    Logout = Server + "/logout?id={0}";

    GetSymbols = Server+ "/getsymbols?id={0}&pattern={1}";
    GetHistory = Server+ "/gethistory?id={0}&symbol={1}&from={2}&to={3}&size={4}&maxbars={5}";
    GetRealtime = Server+ "/getrealtime?id={0}";

    AddRealtime = Server + "/rt_order_symb?id={0}&symbol={1}";
    RemoveRealtime = Server + "/rt_order_symb_remove?id={0}&symbol={1}";

    AddMini = Server + "/rt_mini_symb?id={0}&symbol={1}";
    RemoveMini = Server + "/rt_mini_symb_remove?id={0}&symbol={1}";

#############################################################################################################
# Errors
class HTTPClientError(Exception):
    pass

class ProxyError(Exception):
    pass

